# -*- coding: utf-8 -*-
# encoding:utf-8

import sys
reload(sys)
sys.setdefaultencoding("utf-8")

import redis
import follwer
import time

from red_filter import red2, red_queue
# wrap the class method


def create_new_slave(url, option):
    new_slave = follwer.Follower_Crawler(url, option)
    new_slave.send_request()
    return "ok"


def gevent_worker(option):
    while True:
        url = red.lpop(red_queue)
        if not url:
            break
        create_new_slave(url, option)



if __name__ == "__main__":

    '''
    start the crawler

    '''

    start = time.time()
    count = 0

    # choose the running way of using database or not

    try:
        option = sys.argv[1]
    except:
        option = ''
    if "mongo" not in option:
        option = "print_data_out"
    i = 0
    gevent_worker(option)

    
